#!/usr/bin/python
#
# Copyright 2011 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This code creates new line items from a text file"""

__author__ = 'felipel+bulk@google.com (Felipe Lora)'

# Locate the client library. If module was installed via "setup.py" script, then
# the following two lines are not needed.
import os
import sys
import time
from datetime import *
sys.path.insert(0, os.path.join('..', '..', '..', '..'))

# Import appropriate classes from the client library.
from adspygoogle.common import Utils
from adspygoogle.dfp.DfpClient import DfpClient
from adspygoogle.dfp import DfpUtils
from adspygoogle.dfp.DfpErrors import *
from copy import deepcopy
import pdb

# Initialize client object.
client = DfpClient(path=os.path.join('..', '..', '..', '..'))

# Define the currency of CPC, CPM, CPD
CURRENCY = "EUR"

# Initialize appropriate service. By default, the request is always made against
# the production environment.
inventory_service = client.GetInventoryService('https://www.google.com', 'v201111')
network_service = client.GetNetworkService('https://www.google.com', 'v201111')
order_service = client.GetOrderService('https://www.google.com', 'v201111')
custom_targeting_service = client.GetCustomTargetingService('https://www.google.com', 'v201111')
line_item_service = client.GetLineItemService('https://www.google.com', 'v201111')

pql_service = client.GetPublisherQueryLanguageService('https://www.google.com', 'v201111')

# Global container of network's ad units, orders, custom targeting keys, countries
networkAdUnits = DfpUtils.GetAllEntitiesByStatementWithService(inventory_service)
networkOrders = DfpUtils.GetAllEntitiesByStatementWithService(order_service)

values = [{
    'key': 'type',
    'value': {
        'xsi_type': 'TextValue',
        'value': 'PREDEFINED'
    }
}]
filter_statement = {'query': 'WHERE type = :type LIMIT 500',
                    'values': values}

networkCustomTargetingKeys = custom_targeting_service.GetCustomTargetingKeysByStatement(filter_statement)[0]['results']

country_select_statement = {'query': 'SELECT * FROM Country WHERE targetable = true'}
networkCountries = pql_service.Select(country_select_statement)[0]

# Create indexers for all operations
    
order_indexer = dict((n['name'], i) for i,n in enumerate(networkOrders))
ad_unit_indexer = dict((n['name'], i) for i,n in enumerate(networkAdUnits))
custom_targeting_key_indexer = dict((n['name'], i) for i,n in enumerate(networkCustomTargetingKeys))
networkAdUnitsModified = []

for aU in networkAdUnits:
  tAu = {}
  tAu['id'] = aU['id']
  if 'parentId' in aU:
    tAu['nameParentId'] = str(aU['name']) + str(aU['parentId'])
  else:
    tAu['nameParentId'] = 'root'
  networkAdUnitsModified.append(tAu)

ad_unit_indexer_nameParentId = dict((n['nameParentId'], i) for i,n in enumerate(networkAdUnitsModified)) 

networkCountryCodes = []

for row in networkCountries['rows']:
  c = {
       'code' : row['values'][0]['value'], 
       'id' : row['values'][3]['value'] 
      }
  networkCountryCodes.append(c)

country_indexer = dict((n['code'], i) for i,n in enumerate(networkCountryCodes))


# Get the root network ID
network_root_id = network_service.GetCurrentNetwork()[0]['effectiveRootAdUnitId']

# Define names of logs
warningLog = "warnings_li.log"
errorLog = "errors_li.log"
successLog = "success_li.log"

#############################################################

def getFileLineItems(liFile):

  line_items = []
  lisFile = open(liFile,'rU')
  liValidator = 1

  for line in lisFile.readlines():
    line_item = { }  
    currLine = line.rsplit("^")

    # Convert cost to 'micros'
    # By default we have decided to leave units bought for 1, please change according to your implementation 
    line_item['unitsBought'] = '1'
    line_item['orderId'] = currLine[0]
    line_item['name'] = currLine[1]
    line_item['creativePlaceholders'] = currLine[2].upper()
    line_item['startDate'] = currLine[3]
    line_item['startTime'] = currLine[4]
    line_item['endDate'] = currLine[5]
    line_item['endTime'] = currLine[6]
    line_item['lineItemType'] = currLine[7].upper()
    line_item['priority'] = currLine[8]
    line_item['costType'] = currLine[9]
    line_item['costPerunit'] = currLine[10]
    
    if len(currLine[11].rsplit("\n")) > 1:
      line_item['adUnitTargeting'] = currLine[11].rsplit("\n")[0]
    else:
      line_item['adUnitTargeting'] = currLine[11]

      if len(currLine[12].rsplit("\n")) > 1:
        line_item['customTargeting'] = currLine[12].rsplit("\n")[0]
      else:
        line_item['customTargeting'] = currLine[12]

        if len(currLine[13].rsplit("\n")) > 1:
          line_item['countryTargeting'] = currLine[13].rsplit("\n")[0]
        else:
          line_item['countryTargeting'] = currLine[13]

          if len(currLine[14].rsplit("\n")) > 1:
            line_item['cityTargeting'] = currLine[14].rsplit("\n")[0]
          else:
            line_item['cityTargeting'] = currLine[14]
     
    liValidator = validateLineItem(line_item)
    
    if liValidator == 1:
       line_items.append(line_item)

  return line_items

def validateLineItem(line_item):
  f = open(errorLog,'a')
  flag = 1
  m = ""

  # Validate the length of the name
  if len(line_item['name']) > 127:
    flag = 0
    m = m + "%s, %s, Line item name exceeds the supported 127 characters\n" % (line_item['orderId'], line_item['name'])
  else:
    flag = 1
  
  # Validate that start date is in the future
  now = datetime.now()
  nowYear = int(now.year)
  nowMonth = int(now.month)
  nowDay = int(now.day)
  nowHour = int(now.hour)
  
  liStartYear = int(line_item['startDate'].rsplit('/')[2])
  liStartMonth = int(line_item['startDate'].rsplit('/')[1])
  liStartDay = int(line_item['startDate'].rsplit('/')[0])
  liStartHour = int(line_item['startTime'].rsplit(':')[0])
  
  liEndYear = int(line_item['endDate'].rsplit('/')[2])
  liEndMonth = int(line_item['endDate'].rsplit('/')[1])
  liEndDay = int(line_item['endDate'].rsplit('/')[0])
  liEndHour = int(line_item['endTime'].rsplit(':')[0])
  
  tooEarly = 0

  if liStartYear == nowYear:
    if liStartMonth == nowMonth:
      if liStartDay == nowDay:
        if liStartHour >= nowHour + 2:
          tooEarly = 0
        else:
          tooEarly = 1  
      elif liStartDay < nowDay:
        tooEarly = 1 
    elif liStartMonth < nowMonth:
      tooEarly = 1 
  elif liStartYear < nowYear:
    tooEarly = 1  
  
  if tooEarly == 1:
    flag = 0
    m = m + "%s, %s, Line item start time is in the past\n" % (line_item['orderId'], line_item['name'])

  # Validate that start date is before end date
  forwardTime = 0  
  if liStartYear == liEndYear:
    if liStartMonth == liEndMonth:
      if liStartDay == liEndDay:
        if liStartHour <= liEndHour - 2:
          forwardTime = 0
        else:
          forwardTime = 1  
      elif liStartDay > liEndDay:
        forwardTime = 1 
    elif liStartMonth > liEndMonth:
      forwardTime = 1 
  elif liStartYear > liEndYear:
    forwardTime = 1
    
  if forwardTime == 1:
    flag = 0
    m = m + "%s, %s, Line end date / time is before line item start date / time \n" % (line_item['orderId'], line_item['name'])

  # Validate that priorities match line item types
  # Validate that cost model matches types
  if line_item['lineItemType'] == "SPONSORSHIP":
    if int(line_item['priority']) < 1 or int(line_item['priority']) > 10 :
      m = m + "%s, %s, Priorities for %s must be between 1 and 10\n" % (line_item['orderId'], line_item['name'], line_item['lineItemType'])
      flag = 0
    if int(line_item['priority']) < 1 or int(line_item['priority']) > 10 :
      m = m + "%s, %s, Priorities for %s must be between 1 and 10\n" % (line_item['orderId'], line_item['name'], line_item['lineItemType'])
      flag = 0
    if line_item['costType'] == "CPC" :
      flag = flag
    elif line_item['costType'] == "CPM" :
      flag = flag
    elif line_item['costType'] == "CPD" :
      flag = flag
    else:
      m = m + "%s, %s, Invalid cost type for %s, it should be CPM, CPC or CPD\n" % (line_item['orderId'], line_item['name'], line_item['lineItemType'])
      flag = 0

  elif line_item['lineItemType'] == "STANDARD" :
    if int(line_item['priority']) < 1 or int(line_item['priority']) > 10 :
      m = m + "%s, %s, Priorities for %s must be between 1 and 10\n" % (line_item['orderId'], line_item['name'], line_item['lineItemType'])
      flag = 0
    if line_item['costType'] == "CPC" :
      flag = flag
    elif line_item['costType'] == "CPM" :
      flag = flag
    else:
      m = m + "%s, %s, Invalid cost type for %s, it should be CPM or CPC\n" % (line_item['orderId'], line_item['name'], line_item['lineItemType'])
      flag = 0


  elif line_item['lineItemType'] == "NETWORK" :
    if int(line_item['priority']) < 11 or int(line_item['priority']) > 16 :
      m = m + "%s, %s, Priorities for %s must be between 11 and 16\n" % (line_item['orderId'], line_item['name'], line_item['lineItemType'])
      flag = 0
    if line_item['costType'] == "CPC" :
      flag = flag
    elif line_item['costType'] == "CPM" :
      flag = flag
    elif line_item['costType'] == "CPD" :
      flag = flag
    else:
      m = m + "%s, %s, Invalid cost type for %s, it should be CPM, CPC or CPD\n" % (line_item['orderId'], line_item['name'], line_item['lineItemType'])
      flag = 0


  elif line_item['lineItemType'] == "BULK" :
    if int(line_item['priority']) < 11 or int(line_item['priority']) > 16 :
      m = m + "%s, %s, Priorities for %s must be between 11 and 16\n" % (line_item['orderId'], line_item['name'], line_item['lineItemType'])
      flag = 0
    if line_item['costType'] == "CPC" :
      flag = flag
    elif line_item['costType'] == "CPM" :
      flag = flag
    else:
      m = m + "%s, %s, Invalid cost type for %s, it should be CPM or CPC\n" % (line_item['orderId'], line_item['name'], line_item['lineItemType'])
      flag = 0


  elif line_item['lineItemType'] == "PRICE_PRIORITY" :
    if int(line_item['priority']) < 11 or int(line_item['priority']) > 16 :
      m = m + "%s, %s, Priorities for %s must be between 11 and 16\n" % (line_item['orderId'], line_item['name'], line_item['lineItemType'])
      flag = 0
    if line_item['costType'] == "CPC" :
      flag = flag
    elif line_item['costType'] == "CPM" :
      flag = flag
    else:
      m = m + "%s, %s, Invalid cost type for %s, it should be CPM or CPC\n" % (line_item['orderId'], line_item['name'], line_item['lineItemType'])
      flag = 0


  elif line_item['lineItemType'] == "HOUSE" :
    if int(line_item['priority']) < 11 or int(line_item['priority']) > 16 :
      m = m + "%s, %s, Priorities for %s must be between 11 and 16\n" % (line_item['orderId'], line_item['name'], line_item['lineItemType'])
      flag = 0
    if line_item['costType'] == "CPC" :
      flag = flag
    elif line_item['costType'] == "CPM" :
      flag = flag
    else:
      m = m + "%s, %s, Invalid cost type for %s, it should be CPM or CPC\n" % (line_item['orderId'], line_item['name'], line_item['lineItemType'])
      flag = 0

  elif line_item['lineItemType'] == "CLICK_TRACKING" :
    if line_item['costType'] != "CPC" :
      m = m + "%s, %s, Invalid cost type for %s, it should be CPC\n" % (line_item['orderId'], line_item['name'], line_item['lineItemType'])
      flag = 0

  # Validate that the order exists
  if line_item['orderId'] not in order_indexer:
    flag = 0
    m = "%s,%s, the order assigned to this line item does not exist\n" % (line_item['orderId'], line_item['name'])
  
  f.write(m)
  f.close()
  return flag

def createLIs(lineItems):
  liTemp = deepcopy(lineItems)
  liToCreate = []
  for line_item in liTemp:
    line_item['orderId'] = networkOrders[order_indexer[line_item['orderId']]]['id']
    line_item['creativePlaceholders'] = setSizes(line_item)

    line_item['startDateTime'] = setDate(line_item,'startDate','startTime')
    del(line_item['startDate'])
    del(line_item['startTime'])

    line_item['endDateTime'] = setDate(line_item,'endDate','endTime')
    del(line_item['endDate'])
    del(line_item['endTime'])

    line_item['costPerUnit'] = { 'currencyCode': CURRENCY, 'microAmount': str(int(float(line_item['costPerunit']) * 1000000)) }
    del(line_item['costPerunit'])

    line_item['targeting'] = setTargeting(line_item)
    if 'adUnitTargeting' in line_item:
      del(line_item['adUnitTargeting'])
    if 'customTargeting' in line_item:
      del(line_item['customTargeting'])
    if 'countryTargeting' in line_item:
      del(line_item['countryTargeting'])
    if 'cityTargeting' in line_item:
      del(line_item['cityTargeting'])
    
    try:
      creation = line_item_service.CreateLineItem(line_item)
      successLogFile = open(successLog,'a')
      m = "%s, %s, %s\n" % (creation[0]['id'], creation[0]['orderId'], creation[0]['name'])
      successLogFile.write(m)
      successLogFile.close()
    except DfpRequestError,e:
      errorLogFile = open(errorLog,'a')
      if "UniqueError.NOT_UNIQUE" in e.fault_string:
        m = "%s, %s, There is another line item with the same name under the order, the line item was not created\n" % (line_item['orderId'], line_item['name']) 
        errorLogFile.write(m)
      elif "INVENTORY_UNIT_CANNOT_BE_TARGETED_IF_ANCESTOR_IS_TARGETED" in e.fault_string:
        m = "%s, %s, Ad unit targeting includes a parent and child ad unit of the same tree, the line item has not been created\n" % (line_item['orderId'], line_item['name']) 
      errorLogFile.close()



def setSizes(line_item):
  creativePlaceholders = []

  if len(line_item['creativePlaceholders']) > 1:
    sizes = line_item['creativePlaceholders'].rsplit(",")
  else:
    f = open(warningLog,'a')
    m = "%s,%s,This line item does not have any size, it will be created with a 1x1 \n" % (str(line_item['orderId']), line_item['name'])
    f.write(m)
    f.close()
    temp = {
      'size': {
        'width': 1,
        'height': 1
      }
    }
    return temp
    
 
  for s in sizes:
    widthHeight = s.rsplit("X")
    if s != '':  
      if widthHeight[0].isdigit() and widthHeight[1].isdigit():
        temp = {
          'size': {
            'width': widthHeight[0],
            'height': widthHeight[1]
          }
         }
        creativePlaceholders.append(temp)
      else: 
        f = open(warningLog,'a')
        m = "%s,%s,This line item will be created without the size %s x %s as it is invalid\n" % (str(line_item['orderId']), line_item['name'], str(widthHeight[0]), str(widthHeight[1]))
        f.write(m)
        f.close()
  if len(creativePlaceholders) > 0:  
    return creativePlaceholders
  else:
  # If no valid sizes, instead of blocking the whole line item creation, create it with a 1x1 size
    temp = {
      'size': {
        'width': 1,
        'height': 1
      }
    }
    return temp

def setDate(line_item, whichDate, whichTime):
  date = {
          'date': {
           'year': line_item[whichDate].rsplit('/')[2],
           'month': line_item[whichDate].rsplit('/')[1],
           'day': line_item[whichDate].rsplit('/')[0]        
          },
          'hour': line_item[whichTime].rsplit(':')[0], 
          'minute': line_item[whichTime].rsplit(':')[1] ,
          'second': '0'
         }
  return date

def setTargeting(line_item):
  invTargeting = setInventoryTargeting(line_item)
  cusTargeting = setCustomTargeting(line_item)
  geoTargeting = setGeoTargeting(line_item)
  result = { 
             'inventoryTargeting' : invTargeting,
             'customTargeting' : cusTargeting,
             'geoTargeting' : geoTargeting
           }
  return result

def setInventoryTargeting(line_item):
  targetedAdUnits = []
  ad_unit_tree = []
  auLines = line_item['adUnitTargeting'].rsplit("&")
  for auLine in auLines:
    names = auLine.rsplit(",")
    parentId = network_root_id
    for parentAU in names:
      error = 0
      if parentAU == "RON":
        currId = parentId
        break
      if parentAU in ad_unit_indexer:
        currId = networkAdUnitsModified[ad_unit_indexer_nameParentId[parentAU + parentId]]['id']
      else:
        error = 1
        break
      parentId = currId 
    if error == 0:
      adunitV = { 'adUnitId' : currId }
      targetedAdUnits.append(adunitV)
    else:
      f = open(warningLog,'a')
      m = "%s, %s, The targeted ad unit %s does not exist, it was not included in targeting\n" % (str(line_item['orderId']), line_item['name'], str(names))
      f.write(m)
      f.close()
  response = { 'targetedAdUnits': targetedAdUnits }
  return response

def setGeoTargeting(line_item):
  targetedLocations = []
  valid = 1
  f = open(warningLog, 'a')
  m = ''
  if ('countryTargeting' not in line_item) or (line_item['countryTargeting'] == ''):
    if ('cityTargeting' not in line_item) or (line_item['cityTargeting'] == ''):
      valid = 0

    #Do city
    else:
      citySet = line_item['cityTargeting'].rsplit(',')
      for city in citySet:
        country = city.rsplit('|')[1]
        currCity = city.rsplit('|')[0]
        if country not in country_indexer:
          m = "%s, %s, The country \'%s\' for the city \'%s\'is not a recognized one, it was not included in the targeting\n" % (str(line_item['orderId']), line_item['name'], str(country), str(currCity))
          f.write(m)
        else:
          searchTerm = 'SELECT id FROM City WHERE targetable = true AND cityName = \'' + currCity + '\' AND countrycode = \'' + country + '\'' 
          select_statement = {'query': searchTerm }
          result_set = pql_service.Select(select_statement)[0]
          toAppend = { 'id' : result_set['rows'][0]['values'][0]['value']}
          targetedLocations.append(toAppend)              
       
  # Do country and check for city
  else:
    countrySet = line_item['countryTargeting'].rsplit(',')
    for country in countrySet:
       if country not in country_indexer:
         m = "%s, %s, The country \'%s\' is not a recognized one, it was not included in the targeting\n" % (str(line_item['orderId']), line_item['name'], str(country))
         f.write(m)
       else:
         toAppend = { 'id' : networkCountryCodes[country_indexer[country]]['id'] }
         targetedLocations.append(toAppend)


    if ('cityTargeting' not in line_item) or (line_item['cityTargeting'] == ''):
       valid = 0
    #Do city
    else:
      citySet = line_item['cityTargeting'].rsplit(',')
      for city in citySet:
        country = city.rsplit('|')[1]
        currCity = city.rsplit('|')[0]
        if country not in country_indexer:
          m = "%s, %s, The country \'%s\' for the city \'%s\'is not a recognized one, it was not included in the targeting\n" % (str(line_item['orderId']), line_item['name'], str(country), str(currCity))
          f.write(m)
        else:
          searchTerm = 'SELECT id FROM City WHERE targetable = true AND cityName = \'' + currCity + '\' AND countrycode = \'' + country + '\'' 
          select_statement = {'query': searchTerm }
          result_set = pql_service.Select(select_statement)[0]
          toAppend = { 'id' : result_set['rows'][0]['values'][0]['value']}
          targetedLocations.append(toAppend)            

  f.close()
  geoTargeting = { 'targetedLocations': targetedLocations }
  return geoTargeting 

def setCustomTargeting(line_item):
  keys = []
  valid = 1
  m = ''

  if 'customTargeting' not in line_item:
    return ''

  orCcSets = []
  orSets = line_item['customTargeting'].rsplit('|')
  for orSet in orSets:
    if "&" in orSet:
      andSets = orSet.rsplit("&")
      andCcSets = []
      for andSet in andSets:
        key,values = getKeyValues(andSet,line_item)
        if (key != '') and values:
          cc = { 
                'xsi_type' : 'CustomCriteria',
                'keyId' : key,
                'valueIds' : values,
                'operator' : 'IS'
               }
          andCcSets.append(cc)
      if andCcSets:
        sub_set = {
                   'xsi_type' : 'CustomCriteriaSet',
                   'logicalOperator' : 'AND',
                   'children' : andCcSets
                  }
        orCcSets.append(sub_set)
    else:
      key,values = getKeyValues(orSet,line_item)
      if (key != '') and values:
        cc = { 
              'xsi_type' : 'CustomCriteria',
              'keyId' : key,
              'valueIds' : values,
              'operator' : 'IS'
             }
        orCcSets.append(cc)

  if orCcSets:
    top_set = {
               'xsi_type' : 'CustomCriteriaSet',
               'logicalOperator' : 'OR',
               'children' : orCcSets
              }
    return top_set
  else:
    return ''    

# Return the key ID and the values' IDs. Return empty string for key or empty value lists if no valid values exist
def getKeyValues(expressionSet,line_item):
  s = expressionSet.rsplit("=")
  f = open(warningLog, 'a')
  m = ''
  key = ''
  returnedValues = []

  if s[0] not in custom_targeting_key_indexer:
    m = "%s, %s, The key \'%s\' is not a predefined key in your network, it was not included in the targeting\n" % (str(line_item['orderId']), line_item['name'], s[0])
    f.write(m)
    f.close()
    return key,returnedValues

  elif len(s[1]) == 0:
    m = "%s, %s, The key \'%s\' does not have any values, it was not included in the targeting\n" % (str(line_item['orderId']), line_item['name'], s[0])
    f.write(m)
    f.close()
    return key,returnedValues

  else:
    key = networkCustomTargetingKeys[custom_targeting_key_indexer[s[0]]]['id']
    values = s[1].rsplit(",")
    networkCustomTargetingValues = [{
                                     'key': 'keyId',
                                     'value': {
                                               'xsi_type': 'NumberValue',
                                               'value': key
                                              }
                                    }]
    filter_statement = {'query': 'WHERE customTargetingKeyId = :keyId LIMIT 500','values': networkCustomTargetingValues}
    response = custom_targeting_service.GetCustomTargetingValuesByStatement(filter_statement)[0]
    if 'results' in response:
      keyValues = response['results']
    key_values_indexer = dict((n['name'], i) for i,n in enumerate(keyValues)) 

    for v in values:
      if v not in key_values_indexer:
        m = m + "%s, %s, There is no value \'%s\' predefined for the key \'%s\' therefore it was not included in the targeting\n" % (str(line_item['orderId']), line_item['name'], v, s[0])
      else:
        returnedValues.append(keyValues[key_values_indexer[v]]['id']) 

    f.write(m)
    f.close()
    return key,returnedValues

def Main():
  # Clean old logs
  f = open(warningLog,'w')
  f.write("Order Name, Name, Message\n")
  f.close()
  f = open(errorLog,'w')
  f.write("Order Name, Name,Message\n")
  f.close()
  f = open(successLog,'w')
  f.write("ID, Order Name, Name\n")
  f.close()

  # Read the ad units from the text file
  args = sys.argv[1:]
  liFile = args[0]
  lineItems = getFileLineItems(liFile)
  # Create the line items
  if len(lineItems) != 0:
    print "%s line items loaded to create" % str(len(lineItems))
    createLIs(lineItems)

if __name__ == '__main__':
  Main()
 

